Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

time-on-page: query (#5159)

Karl-Aksel Puulmanne9b30e0ba553
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts
index a52c60da4a..e568c0c31a 100644
--- a/assets/js/types/query-api.d.ts
+++ b/assets/js/types/query-api.d.ts
@@ -201,6 +201,7 @@ export interface QueryApiSchema {
*/
date_range: [string, string];
};
+ legacy_time_on_page_cutoff?: string;
};
pagination?: {
/**
diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.ex
index 409f226c2a..a9fae4ca90 100644
--- a/extra/lib/plausible/stats/funnel.ex
+++ b/extra/lib/plausible/stats/funnel.ex
@@ -27,10 +27,10 @@ defmodule Plausible.Stats.Funnel do
end
end
- def funnel(site, query, %Funnel{} = funnel) do
+ def funnel(_site, query, %Funnel{} = funnel) do
funnel_data =
- site
- |> Base.base_event_query(query)
+ query
+ |> Base.base_event_query()
|> query_funnel(funnel)
# Funnel definition steps are 1-indexed, if there's index 0 in the resulting query,
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index d3ad7bffa0..bc80949539 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -182,7 +182,7 @@ defmodule Plausible.Imported.CSVImporter do
"imported_operating_systems" =>
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_pages" =>
- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64",
+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64, total_time_on_page UInt64, total_time_on_page_visits UInt64",
"imported_sources" =>
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" =>
diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex
index 90709f17d2..cc03cda51b 100644
--- a/lib/plausible/imported/google_analytics4.ex
+++ b/lib/plausible/imported/google_analytics4.ex
@@ -205,7 +205,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently
exits: 0,
- time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ total_time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
+ total_time_on_page_visits: row.metrics |> Map.fetch!("sessions") |> parse_number()
}
end
diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.ex
index 6e27c1bd89..bc4a9edba7 100644
--- a/lib/plausible/imported/page.ex
+++ b/lib/plausible/imported/page.ex
@@ -14,8 +14,9 @@ defmodule Plausible.Imported.Page do
field :active_visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
- field :time_on_page, Ch, type: "UInt64"
field :total_scroll_depth, Ch, type: "UInt64"
field :total_scroll_depth_visits, Ch, type: "UInt64"
+ field :total_time_on_page, Ch, type: "UInt64"
+ field :total_time_on_page_visits, Ch, type: "UInt64"
end
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index de4a9ba3d3..5b77b8c9a5 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -56,7 +56,7 @@ defmodule Plausible.Stats.Aggregate do
|> maybe_round_value(metric)
end
- @metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
+ @metrics_to_round [:bounce_rate, :visit_duration, :sample_percent]
defp maybe_round_value(nil, _metric), do: nil
defp maybe_round_value(value, metric) when metric in @metrics_to_round, do: round(value)
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index e9c2a48b63..efed82b903 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -4,13 +4,13 @@ defmodule Plausible.Stats.Base do
alias Plausible.Stats.{TableDecider, SQL}
import Ecto.Query
- def base_event_query(site, query) do
- events_q = query_events(site, query)
+ def base_event_query(query) do
+ events_q = query_events(query)
if TableDecider.events_join_sessions?(query) do
sessions_q =
from(
- s in query_sessions(site, query),
+ s in query_sessions(query),
select: %{session_id: s.session_id},
where: s.sign == 1,
group_by: s.session_id
@@ -26,7 +26,7 @@ defmodule Plausible.Stats.Base do
end
end
- defp query_events(_site, query) do
+ defp query_events(query) do
q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, query))
on_ee do
@@ -36,7 +36,7 @@ defmodule Plausible.Stats.Base do
q
end
- def query_sessions(_site, query) do
+ def query_sessions(query) do
q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, query))
on_ee do
diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex
index 932e5079f6..8190d9caec 100644
--- a/lib/plausible/stats/custom_props.ex
+++ b/lib/plausible/stats/custom_props.ex
@@ -13,7 +13,7 @@ defmodule Plausible.Stats.CustomProps do
[key]
_ ->
- from(e in base_event_query(site, query),
+ from(e in base_event_query(query),
join: meta in fragment("meta"),
hints: "ARRAY",
on: true,
diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index f3134f198f..91e9149780 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -14,7 +14,7 @@ defmodule Plausible.Stats.FilterSuggestions do
q =
from(
- e in query_sessions(site, query),
+ e in query_sessions(query),
group_by: e.country_code,
order_by: [desc: fragment("count(*)")],
select: e.country_code
@@ -35,7 +35,7 @@ defmodule Plausible.Stats.FilterSuggestions do
def filter_suggestions(site, query, "region", "") do
from(
- e in query_sessions(site, query),
+ e in query_sessions(query),
group_by: e.subdivision1_code,
order_by: [desc: fragment("count(*)")],
select: e.subdivision1_code,
@@ -67,7 +67,7 @@ defmodule Plausible.Stats.FilterSuggestions do
q =
from(
- e in query_sessions(site, query),
+ e in query_sessions(query),
group_by: e.subdivision1_code,
order_by: [desc: fragment("count(*)")],
select: e.subdivision1_code,
@@ -105,7 +105,7 @@ defmodule Plausible.Stats.FilterSuggestions do
def filter_suggestions(site, query, "city", "") do
from(
- e in query_sessions(site, query),
+ e in query_sessions(query),
group_by: e.city_geoname_id,
order_by: [desc: fragment("count(*)")],
select: e.city_geoname_id,
@@ -129,7 +129,7 @@ defmodule Plausible.Stats.FilterSuggestions do
q =
from(
- e in query_sessions(site, query),
+ e in query_sessions(query),
group_by: e.city_geoname_id,
order_by: [desc: fragment("count(*)")],
select: e.city_geoname_id,
@@ -163,7 +163,7 @@ defmodule Plausible.Stats.FilterSuggestions do
def filter_suggestions(site, query, "prop_key", filter_search) do
filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
- from(e in base_event_query(site, query),
+ from(e in base_event_query(query),
join: meta in "meta",
hints: "ARRAY",
on: true,
@@ -213,9 +213,9 @@ defmodule Plausible.Stats.FilterSuggestions do
base_q =
if filter_name in [:pathname, :hostname] do
- base_event_query(site, query)
+ base_event_query(query)
else
- query_sessions(site, query)
+ query_sessions(query)
end
from(e in base_q,
@@ -237,22 +237,19 @@ defmodule Plausible.Stats.FilterSuggestions do
|> wrap_suggestions()
end
- def custom_prop_value_filter_suggestions(site, query, prop_key, filter_search) do
+ def custom_prop_value_filter_suggestions(_site, query, prop_key, filter_search) do
filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
none_q =
from(
- e in base_event_query(
- site,
- Query.remove_top_level_filters(query, ["event:props:#{prop_key}"])
- ),
+ e in base_event_query(Query.remove_top_level_filters(query, ["event:props:#{prop_key}"])),
select: "(none)",
where: not has_key(e, :meta, ^prop_key),
limit: 1
)
search_q =
- from(e in base_event_query(site, query),
+ from(e in base_event_query(query),
select: get_by_key(e, :meta, ^prop_key),
where:
has_key(e, :meta, ^prop_key) and
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index 392e375287..b8af4e9bf9 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -14,7 +14,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
imports_meta: false,
time_labels: false,
total_rows: false,
- comparisons: nil
+ comparisons: nil,
+ legacy_time_on_page_cutoff: nil
}
@default_pagination %{
diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex
index e0843cf1eb..2cc20e0808 100644
--- a/lib/plausible/stats/goal_suggestions.ex
+++ b/lib/plausible/stats/goal_suggestions.ex
@@ -41,7 +41,7 @@ defmodule Plausible.Stats.GoalSuggestions do
query = Query.from(site, params)
native_q =
- from(e in base_event_query(site, query),
+ from(e in base_event_query(query),
where: fragment("? ilike ?", e.name, ^matches),
where: e.name not in ["pageview", "engagement"],
where: fragment("trim(?)", e.name) != "",
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index af7f874be3..6948879252 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -226,7 +226,7 @@ defmodule Plausible.Stats.Imported do
imported_q =
site
|> Imported.Base.query_imported(query)
- |> select_imported_metrics(query.metrics)
+ |> select_imported_metrics(query)
|> paginate_optimization(query)
from(
@@ -234,7 +234,7 @@ defmodule Plausible.Stats.Imported do
cross_join: i in subquery(imported_q),
select: %{}
)
- |> select_joined_metrics(query.metrics)
+ |> select_joined_metrics(query)
end
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query) do
@@ -253,7 +253,7 @@ defmodule Plausible.Stats.Imported do
i.name
)
})
- |> select_imported_metrics(query.metrics)
+ |> select_imported_metrics(query)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
@@ -282,10 +282,10 @@ defmodule Plausible.Stats.Imported do
|> select_merge_as([_i, index], %{
dim0: fragment("CAST(?, 'UInt64')", index)
})
- |> select_imported_metrics(query.metrics)
+ |> select_imported_metrics(query)
end)
|> Enum.reduce(q, fn imports_q, q ->
- naive_dimension_join(q, imports_q, query.metrics)
+ naive_dimension_join(q, imports_q, query)
end)
end
@@ -298,7 +298,7 @@ defmodule Plausible.Stats.Imported do
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(query)
- |> select_imported_metrics(query.metrics)
+ |> select_imported_metrics(query)
|> paginate_optimization(query)
from(s in subquery(q),
@@ -307,7 +307,7 @@ defmodule Plausible.Stats.Imported do
select: %{}
)
|> select_joined_dimensions(query)
- |> select_joined_metrics(query.metrics)
+ |> select_joined_metrics(query)
else
q
end
@@ -319,7 +319,7 @@ defmodule Plausible.Stats.Imported do
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end
- defp naive_dimension_join(q1, q2, metrics) do
+ defp naive_dimension_join(q1, q2, query) do
from(a in subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
@@ -328,7 +328,7 @@ defmodule Plausible.Stats.Imported do
|> select_merge_as([a, b], %{
dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
})
- |> select_joined_metrics(metrics)
+ |> select_joined_metrics(query)
end
# Optimization for cases when grouping by a very high cardinality column.
diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.ex
index 95d005e676..b9719677dc 100644
--- a/lib/plausible/stats/imported/sql/expression.ex
+++ b/lib/plausible/stats/imported/sql/expression.ex
@@ -17,16 +17,16 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {table, _}}} = q,
- metrics
+ query
) do
select_clause =
- metrics
- |> Enum.map(&select_metric(&1, table))
+ query.metrics
+ |> Enum.map(&select_metric(&1, table, query))
|> Enum.reduce(%{}, &Map.merge/2)
q
|> select_merge(q, ^select_clause)
- |> filter_pageviews(metrics, table)
+ |> filter_pageviews(query.metrics, table)
end
defp filter_pageviews(q, metrics, table) do
@@ -39,90 +39,116 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
end
end
- defp select_metric(:visitors, _table) do
+ defp select_metric(:visitors, _table, _query) do
wrap_alias([i], %{visitors: sum(i.visitors)})
end
- defp select_metric(:events, "imported_custom_events") do
+ defp select_metric(:events, "imported_custom_events", _query) do
wrap_alias([i], %{events: sum(i.events)})
end
- defp select_metric(:events, _table) do
+ defp select_metric(:events, _table, _query) do
wrap_alias([i], %{events: sum(i.pageviews)})
end
- defp select_metric(:visits, "imported_exit_pages") do
+ defp select_metric(:visits, "imported_exit_pages", _query) do
wrap_alias([i], %{visits: sum(i.exits)})
end
- defp select_metric(:visits, "imported_entry_pages") do
+ defp select_metric(:visits, "imported_entry_pages", _query) do
wrap_alias([i], %{visits: sum(i.entrances)})
end
- defp select_metric(:visits, _table) do
+ defp select_metric(:visits, _table, _query) do
wrap_alias([i], %{visits: sum(i.visits)})
end
- defp select_metric(:pageviews, "imported_custom_events") do
+ defp select_metric(:pageviews, "imported_custom_events", _query) do
wrap_alias([i], %{pageviews: 0})
end
- defp select_metric(:pageviews, _table) do
+ defp select_metric(:pageviews, _table, _query) do
wrap_alias([i], %{pageviews: sum(i.pageviews)})
end
- defp select_metric(:bounce_rate, "imported_pages") do
+ defp select_metric(:bounce_rate, "imported_pages", _query) do
wrap_alias([i], %{bounces: 0, __internal_visits: 0})
end
- defp select_metric(:bounce_rate, "imported_exit_pages") do
+ defp select_metric(:bounce_rate, "imported_exit_pages", _query) do
wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.exits)})
end
- defp select_metric(:bounce_rate, "imported_entry_pages") do
+ defp select_metric(:bounce_rate, "imported_entry_pages", _query) do
wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.entrances)})
end
- defp select_metric(:bounce_rate, _table) do
+ defp select_metric(:bounce_rate, _table, _query) do
wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.visits)})
end
- defp select_metric(:visit_duration, "imported_pages") do
+ defp select_metric(:visit_duration, "imported_pages", _query) do
wrap_alias([i], %{visit_duration: 0})
end
- defp select_metric(:visit_duration, "imported_exit_pages") do
+ defp select_metric(:visit_duration, "imported_exit_pages", _query) do
wrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.exits)})
end
- defp select_metric(:visit_duration, "imported_entry_pages") do
+ defp select_metric(:visit_duration, "imported_entry_pages", _query) do
wrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.entrances)})
end
- defp select_metric(:visit_duration, _table) do
+ defp select_metric(:visit_duration, _table, _query) do
wrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.visits)})
end
- defp select_metric(:views_per_visit, "imported_exit_pages") do
+ defp select_metric(:views_per_visit, "imported_exit_pages", _query) do
wrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.exits)})
end
- defp select_metric(:views_per_visit, "imported_entry_pages") do
+ defp select_metric(:views_per_visit, "imported_entry_pages", _query) do
wrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.entrances)})
end
- defp select_metric(:views_per_visit, _table) do
+ defp select_metric(:views_per_visit, _table, _query) do
wrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})
end
- defp select_metric(:scroll_depth, "imported_pages") do
+ defp select_metric(:scroll_depth, "imported_pages", _query) do
wrap_alias([i], %{
total_scroll_depth: sum(i.total_scroll_depth),
total_scroll_depth_visits: sum(i.total_scroll_depth_visits)
})
end
- defp select_metric(_metric, _table), do: %{}
+ defp select_metric(:time_on_page, "imported_pages", query) do
+ case query.time_on_page_data do
+ %{include_new_metric: false} ->
+ wrap_alias([i], %{
+ total_time_on_page: 0,
+ total_time_on_page_visits: 0
+ })
+
+ %{include_new_metric: true, cutoff: nil} ->
+ wrap_alias([i], %{
+ total_time_on_page: sum(i.total_time_on_page),
+ total_time_on_page_visits: sum(i.total_time_on_page_visits)
+ })
+
+ %{include_new_metric: true, cutoff: cutoff} ->
+ cutoff_date = cutoff |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date()
+
+ wrap_alias([i], %{
+ total_time_on_page:
+ fragment("sumIf(?, ? >= ?)", i.total_time_on_page, i.date, ^cutoff_date),
+ total_time_on_page_visits:
+ fragment("sumIf(?, ? >= ?)", i.total_time_on_page_visits, i.date, ^cutoff_date)
+ })
+ end
+ end
+
+ defp select_metric(_metric, _table, _query), do: %{}
def group_imported_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
@@ -147,7 +173,7 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
end
defp select_group_fields(q, "event:page", key, _query) do
- select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)})
+ select_merge_as(q, [i], %{key => i.page})
end
defp select_group_fields(q, dimension, key, _query)
@@ -267,40 +293,39 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
})
end
- def select_joined_metrics(q, []), do: q
+ def select_joined_metrics(q, query) do
+ select_metrics =
+ query.metrics
+ |> Enum.map(&joined_metric(&1, query))
+ |> Enum.reduce(%{}, &Map.merge/2)
+
+ select_merge(q, ^select_metrics)
+ end
+
# NOTE: Reverse-engineering the native data bounces and total visit
# durations to combine with imported data is inefficient. Instead both
# queries should fetch bounces/total_visit_duration and visits and be
# used as subqueries to a main query that then find the bounce rate/avg
# visit_duration.
- def select_joined_metrics(q, [:visits | rest]) do
- q
- |> select_merge_as([s, i], %{visits: s.visits + i.visits})
- |> select_joined_metrics(rest)
+ defp joined_metric(:visits, _query) do
+ wrap_alias([s, i], %{visits: s.visits + i.visits})
end
- def select_joined_metrics(q, [:visitors | rest]) do
- q
- |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
- |> select_joined_metrics(rest)
+ defp joined_metric(:visitors, _query) do
+ wrap_alias([s, i], %{visitors: s.visitors + i.visitors})
end
- def select_joined_metrics(q, [:events | rest]) do
- q
- |> select_merge_as([s, i], %{events: s.events + i.events})
- |> select_joined_metrics(rest)
+ defp joined_metric(:events, _query) do
+ wrap_alias([s, i], %{events: s.events + i.events})
end
- def select_joined_metrics(q, [:pageviews | rest]) do
- q
- |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
- |> select_joined_metrics(rest)
+ defp joined_metric(:pageviews, _query) do
+ wrap_alias([s, i], %{pageviews: s.pageviews + i.pageviews})
end
- def select_joined_metrics(q, [:views_per_visit | rest]) do
- q
- |> select_merge_as([s, i], %{
+ defp joined_metric(:views_per_visit, _query) do
+ wrap_alias([s, i], %{
views_per_visit:
fragment(
"if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
@@ -313,12 +338,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
s.__internal_visits
)
})
- |> select_joined_metrics(rest)
end
- def select_joined_metrics(q, [:bounce_rate | rest]) do
- q
- |> select_merge_as([s, i], %{
+ defp joined_metric(:bounce_rate, _query) do
+ wrap_alias([s, i], %{
bounce_rate:
fragment(
"if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
@@ -331,12 +354,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
s.__internal_visits
)
})
- |> select_joined_metrics(rest)
end
- def select_joined_metrics(q, [:visit_duration | rest]) do
- q
- |> select_merge_as([s, i], %{
+ defp joined_metric(:visit_duration, _query) do
+ wrap_alias([s, i], %{
visit_duration:
fragment(
"""
@@ -355,34 +376,51 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
i.__internal_visits
)
})
- |> select_joined_metrics(rest)
end
# The final `scroll_depth` gets selected at a later querybuilding step
# (in `Plausible.Stats.SQL.SpecialMetrics.add/3`). But in order to avoid
# having to join with imported data there again, we select the required
# information from imported data here already.
- def select_joined_metrics(q, [:scroll_depth | rest]) do
- q
- |> select_merge_as([s, i], %{
+ defp joined_metric(:scroll_depth, _query) do
+ wrap_alias([s, i], %{
__imported_total_scroll_depth: i.total_scroll_depth,
__imported_total_scroll_depth_visits: i.total_scroll_depth_visits
})
- |> select_joined_metrics(rest)
+ end
+
+ defp joined_metric(:time_on_page, query) do
+ wrap_alias([s, i], %{
+ __internal_total_time_on_page: s.__internal_total_time_on_page + i.total_time_on_page,
+ __internal_total_time_on_page_visits:
+ s.__internal_total_time_on_page_visits + i.total_time_on_page_visits
+ })
+ |> Map.merge(time_on_page_metric(query))
end
# Ignored as it's calculated separately
- def select_joined_metrics(q, [metric | rest])
- when metric in [:conversion_rate, :group_conversion_rate, :percentage] do
- q
- |> select_joined_metrics(rest)
+ defp joined_metric(metric, _query)
+ when metric in [:conversion_rate, :group_conversion_rate, :percentage] do
+ %{}
end
- def select_joined_metrics(q, [metric | rest]) do
- q
- |> select_merge_as([s, i], %{metric => field(s, ^metric)})
- |> select_joined_metrics(rest)
+ defp joined_metric(metric, _query) do
+ wrap_alias([s, i], %{metric => field(s, ^metric)})
end
defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
+
+ defp time_on_page_metric(query) do
+ if query.time_on_page_data.include_legacy_metric do
+ %{}
+ else
+ wrap_alias([], %{
+ time_on_page:
+ time_on_page(
+ selected_as(:__internal_total_time_on_page),
+ selected_as(:__internal_total_time_on_page_visits)
+ )
+ })
+ end
+ end
end
diff --git a/lib/plausible/stats/legacy/time_on_page.ex b/lib/plausible/stats/legacy/time_on_page.ex
index c004b331dc..488df28e1e 100644
--- a/lib/plausible/stats/legacy/time_on_page.ex
+++ b/lib/plausible/stats/legacy/time_on_page.ex
@@ -1,36 +1,74 @@
defmodule Plausible.Stats.Legacy.TimeOnPage do
@moduledoc """
- Calculation methods for legacy time_on_page metric. Note the metric
- has its own limitations and quirks.
+ Calculation methods for `legacy` time_on_page metric. `Legacy` calculation methods
+ are used when site does not have engagement data for the requested dates.
- Closely coupled with Plausible.Stats.QueryRunner.
+ Query `include.legacy_time_on_page_cutoff` is used to determine what time range
+ to use for the legacy time_on_page calculations.
+
+ Legacy metric is not exposed in the public API.
"""
use Plausible.ClickhouseRepo
+ use Plausible.Stats.SQL.Fragments
+
import Ecto.Query
+ import Plausible.Stats.Util
+
+ alias Plausible.Stats.{Base, Filters, Query, SQL}
- alias Plausible.Stats.{Base, Filters, Query, SQL, Util}
+ def can_merge_legacy_time_on_page?(query) do
+ :time_on_page in query.metrics and query.dimensions in [[], ["event:page"]]
+ end
- def calculate(site, query, ch_results) do
- case {:time_on_page in query.metrics, query.dimensions} do
- {true, []} ->
- aggregate_time_on_page(site, query)
+ def merge_legacy_time_on_page(q, query) do
+ if :time_on_page in query.metrics and query.time_on_page_data.include_legacy_metric do
+ q |> merge_legacy_time_on_page(query, query.dimensions)
+ else
+ q
+ end
+ end
- {true, ["event:page"]} ->
- pages =
- Enum.map(ch_results, fn entry -> Map.get(entry, Util.shortname(query, "event:page")) end)
+ defp merge_legacy_time_on_page(q, query, []) do
+ from(
+ e in subquery(q),
+ inner_join: t in subquery(aggregate_time_on_page_q(query)),
+ on: true
+ )
+ |> select_metrics_and_dimensions(query)
+ end
- breakdown_time_on_page(site, query, pages)
+ defp merge_legacy_time_on_page(q, query, ["event:page"]) do
+ from(
+ e in subquery(q),
+ left_join: t in subquery(breakdown_q(query)),
+ on: e.dim0 == t.pathname
+ )
+ |> select_metrics_and_dimensions(query)
+ end
- _ ->
- %{}
- end
+ defp merge_legacy_time_on_page(q, _query, _dimensions) do
+ q
end
- defp aggregate_time_on_page(site, query) do
+ defp select_metrics_and_dimensions(q, query) do
+ q
+ |> select_join_fields(query, List.delete(query.metrics, :time_on_page), e)
+ |> select_join_fields(query, query.dimensions, e)
+ |> select_merge_as([e, t], %{
+ time_on_page:
+ time_on_page(
+ e.__internal_total_time_on_page + t.total_time_on_page,
+ e.__internal_total_time_on_page_visits + t.transition_count
+ )
+ })
+ end
+
+ defp aggregate_time_on_page_q(query) do
windowed_pages_q =
- from e in Base.base_event_query(site, Query.remove_top_level_filters(query, ["event:page"])),
+ from(e in Base.base_event_query(Query.remove_top_level_filters(query, ["event:page"])),
where: e.name != "engagement",
+ where: ^filter_by_cutoff(query),
select: %{
next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
@@ -45,6 +83,7 @@ defmodule Plausible.Stats.Legacy.TimeOnPage do
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
]
]
+ )
event_page_filter = Filters.get_toplevel_filter(query, "event:page")
@@ -61,54 +100,56 @@ defmodule Plausible.Stats.Legacy.TimeOnPage do
avg_time_per_page_transition_q =
from e in Ecto.Query.subquery(timed_page_transitions_q),
- select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)},
+ select: %{
+ avg: fragment("sum(?)/countIf(?)", e.duration, e.transition),
+ duration: fragment("sum(?)", e.duration),
+ transition_count: fragment("countIf(?)", e.transition)
+ },
group_by: e.pathname
- time_on_page_q =
- from e in Ecto.Query.subquery(avg_time_per_page_transition_q),
- select: fragment("avg(ifNotFinite(?,NULL))", e.avg)
-
- %{[] => ClickhouseRepo.one(time_on_page_q, query: query)}
- end
-
- defp breakdown_time_on_page(_site, _query, []) do
- %{}
+ from(
+ e in subquery(avg_time_per_page_transition_q),
+ select: %{
+ total_time_on_page: fragment("sum(?)", e.duration),
+ transition_count: fragment("sum(?)", e.transition_count)
+ }
+ )
end
- defp breakdown_time_on_page(site, query, pages) do
- import Ecto.Query
-
+ defp breakdown_q(query) do
windowed_pages_q =
- from e in Base.base_event_query(
- site,
- Query.remove_top_level_filters(query, ["event:page", "event:props"])
- ),
- where: e.name != "engagement",
- select: %{
- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
- timestamp: e.timestamp,
- pathname: e.pathname,
- session_id: e.session_id
- },
- windows: [
- event_horizon: [
- partition_by: e.session_id,
- order_by: e.timestamp,
- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
- ]
- ]
+ from(
+ e in Base.base_event_query(
+ Query.remove_top_level_filters(query, ["event:page", "event:props"])
+ ),
+ where: e.name != "engagement",
+ where: ^filter_by_cutoff(query),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
+ )
timed_page_transitions_q =
- from e in subquery(windowed_pages_q),
+ from(e in subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
- where: e.pathname in ^pages,
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,
transition: e.next_pathname != e.pathname,
duration: sum(e.next_timestamp - e.timestamp)
}
+ )
no_select_timed_pages_q =
from e in subquery(timed_page_transitions_q),
@@ -116,47 +157,58 @@ defmodule Plausible.Stats.Legacy.TimeOnPage do
date_range = Query.date_range(query)
- timed_pages_q =
- if query.include_imported do
- # Imported page views have pre-calculated values
- imported_timed_pages_q =
- from i in "imported_pages",
- group_by: i.page,
- where: i.site_id == ^site.id,
- where: i.date >= ^date_range.first and i.date <= ^date_range.last,
- where: i.page in ^pages,
- select: %{
- page: i.page,
- time_on_page: sum(i.time_on_page),
- visits: sum(i.pageviews) - sum(i.exits)
- }
-
- timed_pages_q =
- from e in no_select_timed_pages_q,
- select: %{
- page: e.pathname,
- time_on_page: sum(e.duration),
- visits: fragment("countIf(?)", e.transition)
- }
-
- "timed_pages"
- |> with_cte("timed_pages", as: ^timed_pages_q)
- |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)
- |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)
- |> select(
- [t, i],
- {
- fragment("if(empty(?),?,?)", t.page, i.page, t.page),
- (t.time_on_page + i.time_on_page) / (t.visits + i.visits)
+ if query.include_imported do
+ # Imported page views have pre-calculated values
+ imported_timed_pages_q =
+ from(i in "imported_pages",
+ group_by: i.page,
+ where: i.site_id == ^query.site_id,
+ where: i.date >= ^date_range.first and i.date <= ^date_range.last,
+ select: %{
+ page: i.page,
+ time_on_page: sum(i.total_time_on_page),
+ visits: sum(i.pageviews) - sum(i.exits)
}
)
- else
+
+ timed_pages_q =
from e in no_select_timed_pages_q,
- select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)}
- end
+ select: %{
+ page: e.pathname,
+ time_on_page: fragment("sum(?)", e.duration),
+ visits: fragment("countIf(?)", e.transition)
+ }
- timed_pages_q
- |> Plausible.ClickhouseRepo.all(query: query)
- |> Map.new(fn {path, value} -> {[path], value} end)
+ "timed_pages"
+ |> with_cte("timed_pages", as: ^timed_pages_q)
+ |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)
+ |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)
+ |> select(
+ [t, i],
+ %{
+ pathname: fragment("if(empty(?),?,?)", t.page, i.page, t.page),
+ total_time_on_page: t.time_on_page + i.time_on_page,
+ transition_count: t.visits + i.visits
+ }
+ )
+ else
+ from(e in no_select_timed_pages_q,
+ select: %{
+ pathname: e.pathname,
+ total_time_on_page: fragment("sum(?)", e.duration),
+ transition_count: fragment("countIf(?)", e.transition)
+ }
+ )
+ end
+ end
+
+ defp filter_by_cutoff(query) do
+ case query.time_on_page_data do
+ %{include_legacy_metric: true, include_new_metric: true, cutoff: cutoff} ->
+ dynamic([e], e.timestamp < ^cutoff)
+
+ _ ->
+ true
+ end
end
end
diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex
index 11f54827f0..e9d250283c 100644
--- a/lib/plausible/stats/metrics.ex
+++ b/lib/plausible/stats/metrics.ex
@@ -34,6 +34,7 @@ defmodule Plausible.Stats.Metrics do
def default_value(:visit_duration, _query, _dimensions), do: nil
def default_value(:scroll_depth, _query, _dimensions), do: nil
+ def default_value(:time_on_page, _query, _dimensions), do: nil
@float_metrics [
:views_per_visit,
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index f396cf6388..2bbe5e406c 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -26,7 +26,14 @@ defmodule Plausible.Stats.Query do
revenue_warning: nil,
remove_unavailable_revenue_metrics: false,
site_id: nil,
- site_native_stats_start_at: nil
+ site_native_stats_start_at: nil,
+ # Contains information to determine how to combine legacy and new time on page metrics
+ # Old stats API and dashboard uses legacy metric by default, new API uses new metric by default
+ time_on_page_data: %{
+ include_new_metric: false,
+ include_legacy_metric: true,
+ cutoff: ~U[2100-01-01 00:00:00Z]
+ }
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}
diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex
index e3d8798d30..6e590d866d 100644
--- a/lib/plausible/stats/query_optimizer.ex
+++ b/lib/plausible/stats/query_optimizer.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do
"""
use Plausible
- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time}
+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time, Legacy}
@doc """
This module manipulates an existing query, updating it according to business logic.
@@ -49,7 +49,8 @@ defmodule Plausible.Stats.QueryOptimizer do
&add_missing_order_by/1,
&update_time_in_order_by/1,
&extend_hostname_filters_to_visit/1,
- &remove_revenue_metrics_if_unavailable/1
+ &remove_revenue_metrics_if_unavailable/1,
+ &set_time_on_page_data/1
]
end
@@ -185,4 +186,30 @@ defmodule Plausible.Stats.QueryOptimizer do
else
defp remove_revenue_metrics_if_unavailable(query), do: query
end
+
+ defp set_time_on_page_data(query) do
+ if :time_on_page in query.metrics and query.include.legacy_time_on_page_cutoff do
+ {:ok, cutoff, _} =
+ query.include.legacy_time_on_page_cutoff
+ |> DateTime.from_iso8601()
+
+ cutoff =
+ cutoff
+ |> DateTime.shift_zone!("Etc/UTC")
+ |> DateTime.truncate(:second)
+
+ Query.set(
+ query,
+ time_on_page_data: %{
+ include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),
+ include_legacy_metric:
+ DateTime.after?(cutoff, query.utc_time_range.first) and
+ Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),
+ cutoff: if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)
+ }
+ )
+ else
+ query
+ end
+ end
end
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index 41d04bf2d4..fd148efef8 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -17,7 +17,6 @@ defmodule Plausible.Stats.QueryRunner do
Compare,
QueryOptimizer,
QueryResult,
- Legacy,
Metrics,
SQL,
Util,
@@ -46,9 +45,9 @@ defmodule Plausible.Stats.QueryRunner do
end
defp execute_main_query(%__MODULE__{main_query: query, site: site} = runner) do
- {ch_results, time_on_page} = execute_query(query, site)
+ ch_results = execute_query(query, site)
- main_results = build_from_ch(ch_results, query, time_on_page)
+ main_results = build_from_ch(ch_results, query)
runner = struct!(runner, main_results: main_results)
@@ -75,14 +74,8 @@ defmodule Plausible.Stats.QueryRunner do
%__MODULE__{comparison_query: comparison_query, site: site} = runner
) do
if comparison_query do
- {ch_results, time_on_page} = execute_query(comparison_query, site)
-
- comparison_results =
- build_from_ch(
- ch_results,
- comparison_query,
- time_on_page
- )
+ ch_results = execute_query(comparison_query, site)
+ comparison_results = build_from_ch(ch_results, comparison_query)
struct!(runner, comparison_results: comparison_results)
else
@@ -114,25 +107,19 @@ defmodule Plausible.Stats.QueryRunner do
end
defp execute_query(query, site) do
- ch_results =
- query
- |> SQL.QueryBuilder.build(site)
- |> ClickhouseRepo.all(query: query)
-
- time_on_page = Legacy.TimeOnPage.calculate(site, query, ch_results)
-
- {ch_results, time_on_page}
+ query
+ |> SQL.QueryBuilder.build(site)
+ |> ClickhouseRepo.all(query: query)
end
- defp build_from_ch(ch_results, query, time_on_page) do
+ defp build_from_ch(ch_results, query) do
ch_results
|> Enum.map(fn entry ->
dimension_labels = Enum.map(query.dimensions, &dimension_label(&1, entry, query))
%{
dimensions: dimension_labels,
- metrics:
- Enum.map(query.metrics, &get_metric(entry, &1, dimension_labels, query, time_on_page))
+ metrics: Enum.map(query.metrics, &get_metric(entry, &1, dimension_labels, query))
}
end)
end
@@ -153,7 +140,7 @@ defmodule Plausible.Stats.QueryRunner do
end
on_ee do
- defp get_metric(entry, metric, dimensions, query, _time_on_page)
+ defp get_metric(entry, metric, dimensions, query)
when metric in [:average_revenue, :total_revenue] do
value = Map.get(entry, metric)
@@ -161,10 +148,7 @@ defmodule Plausible.Stats.QueryRunner do
end
end
- defp get_metric(_entry, :time_on_page, dimensions, _query, time_on_page),
- do: Map.get(time_on_page, dimensions)
-
- defp get_metric(entry, :events, _dimensions, query, _time_on_page) do
+ defp get_metric(entry, :events, _dimensions, query) do
cond do
"event:goal" in query.dimensions ->
goal = get_dimension_goal(entry, query)
@@ -184,7 +168,7 @@ defmodule Plausible.Stats.QueryRunner do
end
end
- defp get_metric(entry, metric, _dimensions, _query, _time_on_page), do: Map.get(entry, metric)
+ defp get_metric(entry, metric, _dimensions, _query), do: Map.get(entry, metric)
defp get_dimension_goal(entry, query) do
goal_index = Map.get(entry, Util.shortname(query, "event:goal"))
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index 4708d8027f..22d8af1dc3 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -189,32 +189,32 @@ defmodule Plausible.Stats.SQL.Expression do
def select_dimension(q, key, "visit:city_name", _table, _query),
do: select_merge_as(q, [t], %{key => t.city_name})
- def event_metric(:pageviews) do
+ def event_metric(:pageviews, _query) do
wrap_alias([e], %{
pageviews: scale_sample(fragment("countIf(? = 'pageview')", e.name))
})
end
- def event_metric(:events) do
+ def event_metric(:events, _query) do
wrap_alias([e], %{
events: scale_sample(fragment("countIf(? != 'engagement')", e.name))
})
end
- def event_metric(:visitors) do
+ def event_metric(:visitors, _query) do
wrap_alias([e], %{
visitors: scale_sample(fragment("uniq(?)", e.user_id))
})
end
- def event_metric(:visits) do
+ def event_metric(:visits, _query) do
wrap_alias([e], %{
visits: scale_sample(fragment("uniq(?)", e.session_id))
})
end
on_ee do
- def event_metric(:total_revenue) do
+ def event_metric(:total_revenue, _query) do
wrap_alias(
[e],
%{
@@ -224,7 +224,7 @@ defmodule Plausible.Stats.SQL.Expression do
)
end
- def event_metric(:average_revenue) do
+ def event_metric(:average_revenue, _query) do
wrap_alias(
[e],
%{
@@ -235,20 +235,81 @@ defmodule Plausible.Stats.SQL.Expression do
end
end
- def event_metric(:sample_percent) do
+ def event_metric(:sample_percent, _query) do
wrap_alias([], %{
sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
})
end
- def event_metric(:percentage), do: %{}
- def event_metric(:conversion_rate), do: %{}
- def event_metric(:scroll_depth), do: %{}
- def event_metric(:group_conversion_rate), do: %{}
- def event_metric(:total_visitors), do: %{}
+ def event_metric(:percentage, _query), do: %{}
+ def event_metric(:conversion_rate, _query), do: %{}
+ def event_metric(:scroll_depth, _query), do: %{}
+ def event_metric(:group_conversion_rate, _query), do: %{}
+ def event_metric(:total_visitors, _query), do: %{}
+
+ def event_metric(:time_on_page, query) do
+ selected =
+ case query.time_on_page_data do
+ %{include_new_metric: false} ->
+ wrap_alias(
+ [e],
+ %{
+ __internal_total_time_on_page: 0,
+ __internal_total_time_on_page_visits: 0
+ }
+ )
+
+ %{include_new_metric: true, cutoff: nil} ->
+ wrap_alias(
+ [e],
+ %{
+ __internal_total_time_on_page: fragment("sum(?) / 1000", e.engagement_time),
+ __internal_total_time_on_page_visits:
+ fragment("uniqIf(?, ? = 'engagement')", e.session_id, e.name)
+ }
+ )
+
+ %{include_new_metric: true, cutoff: cutoff} ->
+ wrap_alias(
+ [e],
+ %{
+ __internal_total_time_on_page:
+ fragment(
+ "sumIf(?, ? >= ?) / 1000",
+ e.engagement_time,
+ e.timestamp,
+ ^cutoff
+ ),
+ __internal_total_time_on_page_visits:
+ fragment(
+ "uniqIf(?, ? = 'engagement' and ? >= ?)",
+ e.session_id,
+ e.name,
+ e.timestamp,
+ ^cutoff
+ )
+ }
+ )
+ end
+
+ if query.time_on_page_data.include_legacy_metric do
+ selected
+ else
+ Map.merge(
+ selected,
+ wrap_alias([e], %{
+ time_on_page:
+ time_on_page(
+ selected_as(:__internal_total_time_on_page),
+ selected_as(:__internal_total_time_on_page_visits)
+ )
+ })
+ )
+ end
+ end
- def event_metric(unknown), do: raise("Unknown metric: #{unknown}")
+ def event_metric(unknown, _query), do: raise("Unknown metric: #{unknown}")
def session_metric(:bounce_rate, query) do
# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
diff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.ex
index 8e39fb823f..36ce9d7990 100644
--- a/lib/plausible/stats/sql/fragments.ex
+++ b/lib/plausible/stats/sql/fragments.ex
@@ -154,6 +154,17 @@ defmodule Plausible.Stats.SQL.Fragments do
def meta_value_column(:meta), do: :"meta.value"
def meta_value_column(:entry_meta), do: :"entry_meta.value"
+ defmacro time_on_page(total_time_on_page, total_time_on_page_visits) do
+ quote do
+ fragment(
+ "if(? > 0, toInt32(round((?) / (?))), NULL)",
+ unquote(total_time_on_page_visits),
+ unquote(total_time_on_page),
+ unquote(total_time_on_page_visits)
+ )
+ end
+ end
+
@doc """
Convenience Ecto macro for wrapping a map passed to select_merge_as such that each
expression gets wrapped in dynamic and set as selected_as.
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index 6696f19197..1449447e08 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -10,6 +10,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
alias Plausible.Stats.{Query, QueryOptimizer, TableDecider, SQL}
alias Plausible.Stats.SQL.Expression
+ alias Plausible.Stats.Legacy.TimeOnPage
require Plausible.Stats.SQL.Expression
@@ -50,6 +51,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> build_group_by(:events, events_query)
|> merge_imported(site, events_query)
|> SQL.SpecialMetrics.add(site, events_query)
+ |> TimeOnPage.merge_legacy_time_on_page(events_query)
end
defp join_sessions_if_needed(q, query) do
@@ -124,7 +126,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
defp select_event_metrics(query) do
query.metrics
- |> Enum.map(&SQL.Expression.event_metric/1)
+ |> Enum.map(&SQL.Expression.event_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 8ade1c18a5..4951cc0944 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -16,7 +16,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
|> maybe_add_percentage_metric(site, query)
|> maybe_add_global_conversion_rate(site, query)
|> maybe_add_group_conversion_rate(site, query)
- |> maybe_add_scroll_depth(site, query)
+ |> maybe_add_scroll_depth(query)
end
defp maybe_add_percentage_metric(q, site, query) do
@@ -127,10 +127,10 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
end
end
- def maybe_add_scroll_depth(q, site, query) do
+ def maybe_add_scroll_depth(q, query) do
if :scroll_depth in query.metrics do
max_per_session_q =
- Base.base_event_query(site, query)
+ Base.base_event_query(query)
|> where([e], e.name == "engagement" and e.scroll_depth <= 100)
|> select([e], %{
session_id: e.session_id,
@@ -222,14 +222,14 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
defp total_visitors_subquery(site, query, true = _include_imported) do
wrap_alias([], %{
total_visitors:
- subquery(total_visitors(site, query)) +
+ subquery(total_visitors(query)) +
subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))
})
end
- defp total_visitors_subquery(site, query, false = _include_imported) do
+ defp total_visitors_subquery(_site, query, false = _include_imported) do
wrap_alias([], %{
- total_visitors: subquery(total_visitors(site, query))
+ total_visitors: subquery(total_visitors(query))
})
end
@@ -243,8 +243,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
Query.set(query, filters: totals_query_filters)
end
- defp total_visitors(site, query) do
- Base.base_event_query(site, query)
+ defp total_visitors(query) do
+ Base.base_event_query(query)
|> select([e],
total_visitors: scale_sample(fragment("uniq(?)", e.user_id))
)
diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex
index 5f40a32645..7a936630f1 100644
--- a/lib/plausible/stats/table_decider.ex
+++ b/lib/plausible/stats/table_decider.ex
@@ -111,11 +111,11 @@ defmodule Plausible.Stats.TableDecider do
defp metric_partitioner(_, :pageviews), do: :event
defp metric_partitioner(_, :events), do: :event
defp metric_partitioner(_, :bounce_rate), do: :session
+ defp metric_partitioner(_, :time_on_page), do: :event
defp metric_partitioner(_, :visit_duration), do: :session
defp metric_partitioner(_, :views_per_visit), do: :session
# Calculated metrics - handled on callsite separately from other metrics.
- defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other
# Sample percentage is included in both tables if queried.
defp metric_partitioner(_, :sample_percent), do: :sample_percent
diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex
index 7fc17e0e64..417f42a643 100644
--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex
+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex
@@ -11,6 +11,8 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
case Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do
{:ok, query} ->
+ query = update_time_on_page_query_data(query)
+
results = Plausible.Stats.query(site, query)
json(conn, results)
@@ -24,4 +26,18 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
def schema(conn, _params) do
json(conn, Plausible.Stats.JSONSchema.raw_public_schema())
end
+
+ defp update_time_on_page_query_data(query) do
+ if is_nil(query.include.legacy_time_on_page_cutoff) do
+ Query.set(query,
+ time_on_page_data: %{
+ include_new_metric: true,
+ include_legacy_metric: false,
+ cutoff: nil
+ }
+ )
+ else
+ query
+ end
+ end
end
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index 46ae2613d6..5a24d82afb 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -109,6 +109,11 @@
"additionalProperties": false
}
]
+ },
+ "legacy_time_on_page_cutoff": {
+ "$comment": "only :internal",
+ "type": "string",
+ "format": "datetime"
}
}
},
diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs
index dbec23c141..3baa8be38e 100644
--- a/priv/repo/seeds.exs
+++ b/priv/repo/seeds.exs
@@ -228,12 +228,12 @@ native_stats_range
event = Keyword.merge(event, timestamp: timestamp)
- event =
+ to_insert =
cond do
event_index > 0 && :rand.uniform() < 0.1 ->
event
|> Keyword.merge(name: outbound.event_name)
- |> then(&Plausible.Factory.build(:event, &1))
+ |> then(&[Plausible.Factory.build(:event, &1)])
event_index > 0 && :rand.uniform() < 0.05 ->
amount = Decimal.new(:rand.uniform(100))
@@ -244,14 +244,22 @@ native_stats_range
|> Keyword.merge(revenue_source_amount: amount)
|> Keyword.merge(revenue_reporting_currency: "USD")
|> Keyword.merge(revenue_reporting_amount: amount)
- |> then(&Plausible.Factory.build(:event, &1))
+ |> then(&[Plausible.Factory.build(:event, &1)])
true ->
- event
- |> then(&Plausible.Factory.build(:pageview, &1))
+ pageview = Plausible.Factory.build(:pageview, event)
+
+ engagement =
+ Map.merge(pageview, %{
+ name: "engagement",
+ engagement_time: Enum.random(300..10000),
+ scroll_depth: Enum.random(1..100)
+ })
+
+ [engagement, pageview]
end
- [event | events]
+ to_insert ++ events
end)
|> Enum.reverse()
end)
@@ -272,6 +280,8 @@ site_import =
imported_stats_range
|> Enum.flat_map(fn date ->
Enum.flat_map(0..Enum.random(1..50), fn _ ->
+ pages_visits = Enum.random(1..15)
+
[
Plausible.Factory.build(:imported_visitors,
date: date,
@@ -294,10 +304,11 @@ imported_stats_range
date: date,
page: Enum.random(long_random_paths),
visitors: Enum.random(1..10),
- visits: Enum.random(1..15),
+ visits: pages_visits,
pageviews: Enum.random(1..50),
exits: Enum.random(1..10),
- time_on_page: Enum.random(1000..10000)
+ total_time_on_page: Enum.random(1000..10000),
+ total_time_on_page_visits: pages_visits
)
]
end)
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index 81b857d1f0..11317f7fcb 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -809,7 +809,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
# it will allow us to assert on the session metrics as well.
assert Enum.at(results, 2) == %{
"page" => "/",
- "time_on_page" => 17.677262055264585,
+ "time_on_page" => 18,
"visitors" => 371,
"visits" => 212,
"bounce_rate" => 54.0,
@@ -821,7 +821,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"page" => "/znamenitosti-rima-koje-treba-vidjeti/",
- "time_on_page" => 40.0,
+ "time_on_page" => 40,
"visit_duration" => 0.0,
"visitors" => 1,
"visits" => 1
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index a359d829a0..14b3297fa3 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -53,7 +53,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
imports_meta: false,
time_labels: false,
total_rows: false,
- comparisons: nil
+ comparisons: nil,
+ legacy_time_on_page_cutoff: nil
}
def check_success(params, site, expected_result, schema_type \\ :public) do
@@ -875,7 +876,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
imports_meta: false,
time_labels: true,
total_rows: true,
- comparisons: nil
+ comparisons: nil,
+ legacy_time_on_page_cutoff: nil
},
pagination: %{limit: 10_000, offset: 0}
})
@@ -939,7 +941,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
imports: false,
imports_meta: false,
time_labels: false,
- total_rows: false
+ total_rows: false,
+ legacy_time_on_page_cutoff: nil
},
pagination: %{limit: 10_000, offset: 0}
},
@@ -970,7 +973,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
imports: false,
imports_meta: false,
time_labels: false,
- total_rows: false
+ total_rows: false,
+ legacy_time_on_page_cutoff: nil
},
pagination: %{limit: 10_000, offset: 0}
},
@@ -1004,7 +1008,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
imports_meta: false,
imports: false,
time_labels: false,
- total_rows: false
+ total_rows: false,
+ legacy_time_on_page_cutoff: nil
},
pagination: %{limit: 10_000, offset: 0}
},
diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exs
index 81445df176..1ba7768f9b 100644
--- a/test/plausible/stats/table_decider_test.exs
+++ b/test/plausible/stats/table_decider_test.exs
@@ -79,8 +79,8 @@ defmodule Plausible.Stats.TableDeciderTest do
test "other metrics put in its own result" do
query = make_query([])
- assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
- {[], [:percentage], [:time_on_page, :total_visitors]}
+ assert partition_metrics([:percentage, :total_visitors], query) ==
+ {[], [:percentage], [:total_visitors]}
end
test "metrics that can be calculated on either when event-only metrics" do
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
index ae0e05a64a..2ff1477f78 100644
--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
@@ -1,2 +1,2 @@
name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
-/some-other-page,1,1,0,60.0,
+/some-other-page,1,1,0,60,
diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv
index c04e0e7d7c..30fdeb67a1 100644
--- a/test/plausible_web/controllers/CSVs/30d/pages.csv
+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv
@@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
/,4,3,67,,
-/signup,1,1,0,60.0,
-/some-other-page,1,1,0,60.0,
+/signup,1,1,0,60,
+/some-other-page,1,1,0,60,
diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv
index 294a95100a..3e7aadc91d 100644
--- a/test/plausible_web/controllers/CSVs/6m/pages.csv
+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv
@@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
/,5,4,75,,
-/signup,1,1,0,60.0,
-/some-other-page,1,1,0,60.0,
+/signup,1,1,0,60,
+/some-other-page,1,1,0,60,
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 92d4e177d0..9281309050 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -2526,9 +2526,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
)
populate_stats(site, site_import.id, [
- build(:imported_pages, page: "/A", time_on_page: 40, date: ~D[2021-01-01]),
- build(:imported_pages, page: "/A", time_on_page: 110, date: ~D[2021-01-01]),
- build(:imported_pages, page: "/B", time_on_page: 499, date: ~D[2021-01-01]),
+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),
+ build(:imported_pages, page: "/A", total_time_on_page: 110, date: ~D[2021-01-01]),
+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01]),
build(:pageview, pathname: "/A", user_id: 4, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/B", user_id: 4, timestamp: ~N[2021-01-01 00:01:00])
])
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
new file mode 100644
index 0000000000..455be7b9a3
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
@@ -0,0 +1,412 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
+ use PlausibleWeb.ConnCase
+
+ setup [:create_user, :create_site, :create_api_key, :use_api_key, :create_site_import]
+
+ test "aggregated time_on_page metric based on engagement data", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:10:00],
+ engagement_time: 120_000
+ ),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:11:00],
+ engagement_time: 20_000
+ ),
+ build(:pageview, user_id: 13, pathname: "/blog", timestamp: ~N[2021-01-01 00:10:00]),
+ build(:engagement,
+ user_id: 13,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:10:00],
+ engagement_time: 60_000
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/blog"], "metrics" => [100]}
+ ]
+ end
+
+ test "aggregated time_on_page metric with imported data", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:10:00],
+ engagement_time: 120_000
+ ),
+ build(:imported_pages,
+ page: "/blog",
+ date: ~D[2021-01-01],
+ visitors: 9,
+ total_time_on_page: 9 * 20,
+ total_time_on_page_visits: 9
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{"imports" => true}
+ })
+
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/blog"], "metrics" => [30]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true
+ })
+ } = json_response(conn, 200)
+ end
+
+ test "time_on_page time series", %{conn: conn, site: site, site_import: site_import} do
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-01 00:03:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-02 00:03:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 13, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),
+ build(:engagement,
+ user_id: 13,
+ pathname: "/",
+ timestamp: ~N[2021-01-02 00:10:00],
+ engagement_time: 300_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-03 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-03 00:03:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-04 00:03:00],
+ engagement_time: 100_000
+ ),
+ build(:imported_pages,
+ page: "/blog",
+ date: ~D[2021-01-01],
+ visitors: 9,
+ total_time_on_page: 9 * 20,
+ total_time_on_page_visits: 9
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => ["2021-01-01", "2021-01-04"],
+ "dimensions" => ["time:day", "event:page"],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},
+ %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},
+ %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},
+ %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},
+ %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}
+ ]
+ end
+
+ describe "include.legacy_time_on_page_cutoff" do
+ setup %{site: site, site_import: site_import} = context do
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-01 00:02:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:02:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:05:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:05:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-01 00:10:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:10:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2021-01-01 00:15:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 13, pathname: "/pricing", timestamp: ~N[2021-01-02 01:00:00]),
+ build(:engagement,
+ user_id: 13,
+ pathname: "/pricing",
+ timestamp: ~N[2021-01-02 01:02:00],
+ engagement_time: 30_000
+ ),
+ build(:pageview, user_id: 14, pathname: "/", timestamp: ~N[2021-01-02 01:00:00]),
+ build(:engagement,
+ user_id: 14,
+ pathname: "/",
+ timestamp: ~N[2021-01-02 01:02:00],
+ engagement_time: 30_000
+ ),
+ build(:imported_pages,
+ page: "/blog",
+ date: ~D[2021-01-01],
+ visitors: 9,
+ total_time_on_page: 9 * 20,
+ total_time_on_page_visits: 9
+ )
+ ])
+
+ context
+ end
+
+ test "breakdown with cutoff being before data (new time-on-page query used)", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "1970-01-01T00:00:00Z",
+ "imports" => true
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/"], "metrics" => [115]},
+ # (2 * 100s + 9 * 20s) / 10 = 38
+ %{"dimensions" => ["/blog"], "metrics" => [38]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ]
+ end
+
+ test "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",
+ "imports" => true
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/"], "metrics" => [420]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [nil]}
+ ]
+ end
+
+ test "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",
+ "imports" => true
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/"], "metrics" => [225]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ]
+ end
+
+ test "breakdown with cutoff being mid-data (two queries joined), no imports", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/"], "metrics" => [225]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ]
+ end
+
+ test "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/blog"]]],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",
+ "imports" => true
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [180]}
+ ]
+ end
+
+ test "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/blog"]]],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",
+ "imports" => true
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [180]}
+ ]
+ end
+
+ test "aggregation with cutoff being mid-data (two queries joined), no imports", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/blog"]]],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [180]}
+ ]
+ end
+ end
+
+ describe "legacy time_on_page metric" do
+ test "aggregated", %{conn: conn, site: site, site_import: site_import} do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),
+ build(:pageview, pathname: "/A", user_id: 999, timestamp: ~N[2021-01-02 00:00:00]),
+ build(:pageview, pathname: "/B", user_id: 999, timestamp: ~N[2021-01-02 00:01:30]),
+ # These are ignored for time_on_page metric
+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),
+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "time_on_page"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/A"]]],
+ "include" => %{
+ "imports" => true,
+ "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [3, 75]}
+ ]
+ end
+
+ test "breakdown", %{conn: conn, site: site, site_import: site_import} do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),
+ build(:pageview, pathname: "/A", user_id: 999, timestamp: ~N[2021-01-02 00:00:00]),
+ build(:pageview, pathname: "/B", user_id: 999, timestamp: ~N[2021-01-02 00:01:30]),
+ build(:pageview, pathname: "/C", user_id: 999, timestamp: ~N[2021-01-02 00:02:00]),
+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),
+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "time_on_page"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{
+ "imports" => true,
+ "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/A"], "metrics" => [3, 63]},
+ %{"dimensions" => ["/B"], "metrics" => [3, 264]},
+ %{"dimensions" => ["/C"], "metrics" => [1, nil]}
+ ]
+ end
+ end
+end
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 673b091655..58a3562927 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -692,7 +692,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
date: ~D[2020-01-01],
visitors: 3,
pageviews: 3,
- time_on_page: 90,
+ total_time_on_page: 90,
+ total_time_on_page_visits: 3,
page: "/blog",
total_scroll_depth: 120,
total_scroll_depth_visits: 3
@@ -748,7 +749,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
date: ~D[2020-01-01],
visitors: 4,
pageviews: 4,
- time_on_page: 180,
+ total_time_on_page: 180,
+ total_time_on_page_visits: 4,
page: "/native-and-imported",
total_scroll_depth: 120,
total_scroll_depth_visits: 3
@@ -757,7 +759,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
date: ~D[2020-01-01],
visitors: 20,
pageviews: 30,
- time_on_page: 300,
+ total_time_on_page: 300,
+ total_time_on_page_visits: 10,
page: "/imported-only",
total_scroll_depth: 100,
total_scroll_depth_visits: 10
@@ -833,7 +836,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 110,
"pageviews" => 160,
"bounce_rate" => 0,
- "time_on_page" => 0.125,
+ "time_on_page" => 0,
"scroll_depth" => 10
}
]
@@ -1374,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
])
assert [
- %{"name" => "/", "time_on_page" => _three_minutes = 180.0},
+ %{"name" => "/", "time_on_page" => _three_minutes = 180},
%{"name" => "/exit", "time_on_page" => nil}
] =
conn
@@ -1407,9 +1410,9 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
])
assert [
- %{"name" => "/a", "time_on_page" => 100.0},
+ %{"name" => "/a", "time_on_page" => 100},
%{"name" => "/b", "time_on_page" => nil},
- %{"name" => "/d", "time_on_page" => +0.0}
+ %{"name" => "/d", "time_on_page" => 0}
] =
conn
|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
@@ -1439,7 +1442,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
build(:imported_pages,
page: "/",
date: ~D[2021-01-01],
- time_on_page: 700
+ total_time_on_page: 700,
+ total_time_on_page_visits: 3
),
build(:imported_entry_pages,
entry_page: "/",
@@ -1450,7 +1454,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
build(:imported_pages,
page: "/some-other-page",
date: ~D[2021-01-01],
- time_on_page: 60
+ total_time_on_page: 60,
+ total_time_on_page_visits: 1
)
])
@@ -1529,7 +1534,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
page: "/",
visitors: 3,
pageviews: 3,
- time_on_page: 300,
+ total_time_on_page: 300,
+ total_time_on_page_visits: 3,
date: ~D[2021-01-01]
),
build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])
@@ -1574,7 +1580,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
page: "/",
visitors: 3,
pageviews: 3,
- time_on_page: 300,
+ total_time_on_page: 300,
+ total_time_on_page_visits: 3,
date: ~D[2021-01-01]
),
build(:imported_pages,
@@ -1632,7 +1639,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
page: "/aaa",
visitors: 3,
pageviews: 3,
- time_on_page: 300,
+ total_time_on_page: 300,
+ total_time_on_page_visits: 3,
date: ~D[2021-01-01]
),
build(:imported_pages,
@@ -1700,7 +1708,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"comparison" => %{
"bounce_rate" => 0,
"pageviews" => 0,
- "time_on_page" => 0,
+ "time_on_page" => nil,
"visitors" => 0,
"scroll_depth" => nil,
"change" => %{
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 99481f4592..b60c276705 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -276,8 +276,8 @@ defmodule PlausibleWeb.StatsControllerTest do
assert pages == [
["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],
- ["/blog", "3", "4", "33", "60.0", "60"],
- ["/another", "2", "2", "0", "60.0", "25"],
+ ["/blog", "3", "4", "33", "60", "60"],
+ ["/another", "2", "2", "0", "60", "25"],
[""]
]
end
@@ -587,7 +587,7 @@ defmodule PlausibleWeb.StatsControllerTest do
"time_on_page",
"scroll_depth"
],
- ["/test", "1", "1", "0.0", "10.0", ""],
+ ["/test", "1", "1", "0.0", "10", ""],
[""]
]
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 2e1383a6b7..c4478e1b54 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -277,7 +277,8 @@ defmodule Plausible.Factory do
visitors: 1,
pageviews: 1,
exits: 0,
- time_on_page: 10
+ total_time_on_page: 10,
+ total_time_on_page_visits: 1
}
end